
import numpy as np
import pandas as pd


books = pd.read_csv('books_data.csv')
books.head()


books.shape

(212404, 10)


books = books.drop(['image', 'previewLink', 'infoLink'], axis = 1, inplace = False)
print(books.shape)
books.head()

(212404, 7)


import matplotlib.pyplot as plt

# Extract year from publishedDate
books['publishedYear'] = books['publishedDate'].str.extract(r'(\d{4})')

# Drop rows where publishedYear is NaN (optional, depending on how you want to handle missing years)
books = books.dropna(subset=['publishedYear'])

# Convert the extracted year to an integer
books['publishedYear'] = books['publishedYear'].astype(int)

# Display the DataFrame to check the new 'publishedYear' column
books.head()


books.drop('publishedDate', axis=1, inplace=True)
books.head()


# Count the number of books published per year
books_per_year = books['publishedYear'].value_counts().sort_index()
books_per_year


# Plot the number of books published per year using a bar chart
plt.figure(figsize=(14, 6))
plt.bar(books_per_year.index, books_per_year.values, color='skyblue')
plt.title('Number of Books Published Per Year')
plt.xlabel('Year')
plt.ylabel('Number of Books')

# Set x-ticks to be every 10 years
start_year = books_per_year.index.min()
end_year = books_per_year.index.max()
plt.xticks(np.arange(start_year, end_year + 1, 10), rotation=45)
plt.grid(True, axis='y')
plt.show()


books.dropna(inplace=True)


import pandas as pd
import string
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

# Download necessary NLTK data
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('omw-1.4')
nltk.download('stopwords')

def clean_text(text):
    # Convert to lowercase
    text = text.lower()

    # Remove punctuation
    text = text.translate(str.maketrans('', '', string.punctuation))

    # Tokenize the text
    tokens = word_tokenize(text)

    # Lemmatization
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(word) for word in tokens]

    # Rejoin the tokens into a single string
    cleaned_text = ' '.join(tokens)

    return cleaned_text

books['description'] = books['description'].apply(lambda x: clean_text(str(x)) if pd.notnull(x) else x)

[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data] Downloading package omw-1.4 to /root/nltk_data...
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.


# Display the cleaned data
books[['Title', 'description']].head()


import matplotlib.pyplot as plt
from wordcloud import WordCloud

# Preprocess the 'description' column to handle NaN values
books['description'] = books['description'].fillna('')
books['Title'] = books['Title'].fillna('')

# Combine all titles into one string for the word cloud
title_text = ' '.join(books['Title'])

# Combine all descriptions into one string for the word cloud
description_text = ' '.join(books['description'])

# Generate the word cloud for the Title column
title_wordcloud = WordCloud(width=800, height=400, background_color='white').generate(title_text)

# Generate the word cloud for the Description column
description_wordcloud = WordCloud(width=800, height=400, background_color='white').generate(description_text)

# Plot the word cloud for Titles
plt.figure(figsize=(10, 5))
plt.imshow(title_wordcloud, interpolation='bilinear')
plt.title('Word Cloud for Titles')
plt.axis('off')
plt.show()

# Plot the word cloud for Descriptions
plt.figure(figsize=(10, 5))
plt.imshow(description_wordcloud, interpolation='bilinear')
plt.title('Word Cloud for Descriptions')
plt.axis('off')
plt.show()


import matplotlib.pyplot as plt

# Get the top 10 most frequent publishers
top_publishers = books['publisher'].value_counts().head(50)

# Create the bar chart
plt.figure(figsize=(10, 6))
top_publishers.plot(kind='bar')
plt.title('Top 50 Most Frequent Publishers')
plt.xlabel('Publishers')
plt.ylabel('Count')
plt.xticks(rotation=45, ha='right')
plt.tight_layout()
plt.show()


from collections import Counter

category_counts = Counter([item.strip() for sublist in books['categories'].dropna().str[1:-1].str.replace("'", "").str.split(', |& ') for item in sublist])
category_counts_df = pd.DataFrame(category_counts.items(), columns=['Category', 'Count']).sort_values(by='Count', ascending=False)
category_counts_df.head(10)


import re
from collections import Counter

# Function to split by commas but neglect commas inside quotes and handle suffixes
def split_authors(authors_str):
    if authors_str:
        # Use regex to find and split by commas outside of quotes
        return re.split(r',\s*(?!\s*(Jr\.|Sr\.|Ph\.D\.|M\.D\.|D\.D\.S\.|Inc\.|LLC\.|Ltd\.)$)', authors_str)
    return []

# Apply the splitting function, filtering out irrelevant entities like "Inc.", "PhD", etc.
def clean_authors(author_list):
    ignore_list = {'Inc', 'PhD', 'LLC', 'Ltd', 'M.D.', 'D.D.S.', 'Incorporated'}
    return [author for author in author_list if author not in ignore_list]

# Process and count authors
author_counts = Counter([
    item.strip()
    for sublist in books['authors'].dropna().str[1:-1].str.replace("'", "").apply(split_authors)
    for item in clean_authors(sublist) if item
])

# Convert to DataFrame for easier analysis
author_counts_df = pd.DataFrame(author_counts.items(), columns=['Author', 'Count']).sort_values(by='Count', ascending=False).reset_index(drop=True)
author_counts_df


# Create a bar plot for top 25 authors
author_counts_df.head(25).plot(kind='bar', x='Author', y='Count', legend=False, figsize=(12, 5))
plt.xlabel('Author')
plt.ylabel('Count')
plt.title('Top 25 Authors')
plt.xticks(rotation=45, ha='right')
plt.show()

# Create a pie chart for top 15 categories
plt.figure(figsize=(10, 5))
plt.pie(category_counts_df.head(15)['Count'], labels=category_counts_df.head(15)['Category'], autopct='%1.1f%%')
plt.title('Top 15 Categories')
plt.tight_layout()
plt.show()


from nbconvert import HTMLExporter
import nbformat

# Load the notebook
notebook_path = '/content/Books_data_EDA.ipynb'  # Replace with the path to your notebook
with open(notebook_path, 'r') as notebook_file:
    notebook_content = notebook_file.read()

# Convert the notebook to HTML
notebook_node = nbformat.reads(notebook_content, as_version=4)
html_exporter = HTMLExporter()
body, resources = html_exporter.from_notebook_node(notebook_node)

# Save the HTML file
html_path = '/content/Books_data_EDA.html'  # Path to save the HTML file
with open(html_path, 'w') as html_file:
    html_file.write(body)

print(f"HTML file saved as {html_path}")

HTML file saved as /content/Books_data_EDA.html


# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES
# TO THE CORRECT LOCATION (/kaggle/input) IN YOUR NOTEBOOK,
# THEN FEEL FREE TO DELETE THIS CELL.
# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON
# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR
# NOTEBOOK.

import os
import sys
from tempfile import NamedTemporaryFile
from urllib.request import urlopen
from urllib.parse import unquote, urlparse
from urllib.error import HTTPError
from zipfile import ZipFile
import tarfile
import shutil

CHUNK_SIZE = 40960
DATA_SOURCE_MAPPING = 'amazon-books-reviews:https%3A%2F%2Fstorage.googleapis.com%2Fkaggle-data-sets%2F2476732%2F4200454%2Fbundle%2Farchive.zip%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com%252F20240904%252Fauto%252Fstorage%252Fgoog4_request%26X-Goog-Date%3D20240904T070744Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D63734674518450559419f2a4efa8d962e101eae6ff601eb6722e120856e001f20dc34d82f4bc0999088ed7dc13ce77c42629842d3b3cf7b2196d78636f23c7d561bf9d3bb7cf40689029da97f4a4b16786ab44b54cce836526e81fea0583b73d1438bbaeded19dcaf9a3257fe2c952174910758e429f9f54c4debb04872b20c198872786bc559e01cf5dd47528b95e18e4fdfae368e570cf9fa084b2d19da3dc2679ca6c541f6aa1c3be0a6f8b8ebe5e60bd253e635fc4fbdc33baecefdaf2c14716ea11aeb2ade10e789f27bf6f2a0cc2813305474af958146705a9386ec7493c532f5b15051fcd38918378fd4e00e80629735cd757722c0e99009f6b654cf2'

KAGGLE_INPUT_PATH='/kaggle/input'
KAGGLE_WORKING_PATH='/kaggle/working'
KAGGLE_SYMLINK='kaggle'

!umount /kaggle/input/ 2> /dev/null
shutil.rmtree('/kaggle/input', ignore_errors=True)
os.makedirs(KAGGLE_INPUT_PATH, 0o777, exist_ok=True)
os.makedirs(KAGGLE_WORKING_PATH, 0o777, exist_ok=True)

try:
  os.symlink(KAGGLE_INPUT_PATH, os.path.join("..", 'input'), target_is_directory=True)
except FileExistsError:
  pass
try:
  os.symlink(KAGGLE_WORKING_PATH, os.path.join("..", 'working'), target_is_directory=True)
except FileExistsError:
  pass

for data_source_mapping in DATA_SOURCE_MAPPING.split(','):
    directory, download_url_encoded = data_source_mapping.split(':')
    download_url = unquote(download_url_encoded)
    filename = urlparse(download_url).path
    destination_path = os.path.join(KAGGLE_INPUT_PATH, directory)
    try:
        with urlopen(download_url) as fileres, NamedTemporaryFile() as tfile:
            total_length = fileres.headers['content-length']
            print(f'Downloading {directory}, {total_length} bytes compressed')
            dl = 0
            data = fileres.read(CHUNK_SIZE)
            while len(data) > 0:
                dl += len(data)
                tfile.write(data)
                done = int(50 * dl / int(total_length))
                sys.stdout.write(f"\r[{'=' * done}{' ' * (50-done)}] {dl} bytes downloaded")
                sys.stdout.flush()
                data = fileres.read(CHUNK_SIZE)
            if filename.endswith('.zip'):
              with ZipFile(tfile) as zfile:
                zfile.extractall(destination_path)
            else:
              with tarfile.open(tfile.name) as tarfile:
                tarfile.extractall(destination_path)
            print(f'\nDownloaded and uncompressed: {directory}')
    except HTTPError as e:
        print(f'Failed to load (likely expired) {download_url} to path {destination_path}')
        continue
    except OSError as e:
        print(f'Failed to load {download_url} to path {destination_path}')
        continue

print('Data source import complete.')


import numpy as np
import pandas as pd


ratings = pd.read_csv('/kaggle/input/amazon-books-reviews/Books_rating.csv')
ratings.head()


ratings.dtypes

Id                     object
Title                  object
Price                 float64
User_id                object
profileName            object
review/helpfulness     object
review/score          float64
review/time             int64
review/summary         object
review/text            object
dtype: object


ratings.drop(['Id', 'User_id', 'profileName', 'review/time'], axis = 1, inplace=True)
ratings.isnull().sum()

Title                     208
Price                 2518829
review/helpfulness          0
review/score                0
review/summary            407
review/text                 8
dtype: int64


ratings.shape

(3000000, 6)


ratings.dropna(subset=['Title', 'review/summary', 'review/text'], inplace=True)
ratings.isnull().sum()

Title                       0
Price                 2518250
review/helpfulness          0
review/score                0
review/summary              0
review/text                 0
dtype: int64


ratings.shape

(2999377, 6)


from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import re
import numpy as np

# Initialize the PorterStemmer
stemmer = PorterStemmer()

# Preprocessing function
def preprocess_text_vectorized(text_series):
    text_series = text_series.str.lower()
    text_series = text_series.str.replace(r'[^\w\s]', '', regex=True)
    text_series = text_series.str.replace(r'\d+', '', regex=True)
    text_series = text_series.apply(word_tokenize)

    stop_words = set(stopwords.words('english'))

    # Remove stop words and apply stemming
    text_series = text_series.apply(lambda tokens: [stemmer.stem(word) for word in tokens if word not in stop_words])

    return text_series.str.join(' ')

# Split the dataset into chunks (adjust the number of splits as needed)
n_splits = 30  # You can adjust this number based on the size of your data and available memory
chunks = np.array_split(ratings, n_splits)

# Process each chunk and save the result separately
for i, chunk in enumerate(chunks):
    chunk['title'] = preprocess_text_vectorized(chunk['Title'])
    chunk['review/Summary'] = preprocess_text_vectorized(chunk['review/summary'])
    chunk['review/Text'] = preprocess_text_vectorized(chunk['review/text'])

    # Save each processed chunk to a separate CSV file
    chunk.to_csv(f"Cleaned_ratings_chunk_{i}.csv", index=False)
    print(f"Processed and saved chunk {i + 1} of {n_splits}")

# List to hold each chunk DataFrame
processed_chunks = []

# Load each processed chunk and append to the list
for i in range(n_splits):
    chunk = pd.read_csv(f"Cleaned_ratings_chunk_{i}.csv")
    processed_chunks.append(chunk)

# Combine all chunks into a single DataFrame
processed_ratings = pd.concat(processed_chunks, ignore_index=True)

# Save the combined DataFrame
processed_ratings.to_csv("Cleaned_ratings_combined.csv", index=False)

print("All chunks combined and saved as 'Cleaned_ratings_combined.csv'")

/opt/conda/lib/python3.10/site-packages/numpy/core/fromnumeric.py:59: FutureWarning: 'DataFrame.swapaxes' is deprecated and will be removed in a future version. Please use 'DataFrame.transpose' instead.
  return bound(*args, **kwds)

Processed and saved chunk 1 of 30
Processed and saved chunk 2 of 30
Processed and saved chunk 3 of 30
Processed and saved chunk 4 of 30
Processed and saved chunk 5 of 30
Processed and saved chunk 6 of 30
Processed and saved chunk 7 of 30
Processed and saved chunk 8 of 30
Processed and saved chunk 9 of 30
Processed and saved chunk 10 of 30
Processed and saved chunk 11 of 30
Processed and saved chunk 12 of 30
Processed and saved chunk 13 of 30
Processed and saved chunk 14 of 30
Processed and saved chunk 15 of 30
Processed and saved chunk 16 of 30
Processed and saved chunk 17 of 30
Processed and saved chunk 18 of 30
Processed and saved chunk 19 of 30
Processed and saved chunk 20 of 30
Processed and saved chunk 21 of 30
Processed and saved chunk 22 of 30
Processed and saved chunk 23 of 30
Processed and saved chunk 24 of 30
Processed and saved chunk 25 of 30
Processed and saved chunk 26 of 30
Processed and saved chunk 27 of 30
Processed and saved chunk 28 of 30
Processed and saved chunk 29 of 30
Processed and saved chunk 30 of 30
All chunks combined and saved as 'Cleaned_ratings_combined.csv'


import numpy as np
import pandas as pd

ratings = pd.read_csv('/kaggle/working/Cleaned_ratings_combined.csv')
ratings.head()


ratings.drop(['title', 'review/summary', 'review/text', 'review/helpfulness'], axis = 1, inplace = True)
ratings.shape

(2999377, 5)


# Remove duplicate rows
ratings = ratings.drop_duplicates(keep = 'first')

# Save the cleaned DataFrame to a new CSV file (optional)
ratings.to_csv("Cleaned_ratings_no_duplicates.csv", index=False)

# Display the first few rows of the cleaned DataFrame
ratings.head()


ratings.shape

(2647307, 5)


from nbconvert import HTMLExporter
import nbformat

# Load the notebook
notebook_path = '/content/Ratings_data_Preprocess.ipynb'  # Replace with the path to your notebook
with open(notebook_path, 'r') as notebook_file:
    notebook_content = notebook_file.read()

# Convert the notebook to HTML
notebook_node = nbformat.reads(notebook_content, as_version=4)
html_exporter = HTMLExporter()
body, resources = html_exporter.from_notebook_node(notebook_node)

# Save the HTML file
html_path = '/content/Ratings_data_Preprocess.html'  # Path to save the HTML file
with open(html_path, 'w') as html_file:
    html_file.write(body)

print(f"HTML file saved as {html_path}")

HTML file saved as /content/Ratings_data_Preprocess.html


from textblob import TextBlob

def get_sentiment_score(text):
    # Handle NaN values by converting them to empty strings
    if pd.isna(text):
        text = ""
    analysis = TextBlob(text)
    # The sentiment score ranges from -1 (negative) to 1 (positive)
    return analysis.sentiment.polarity


# Apply sentiment analysis to both 'review/Summary' and 'review/Text'
cleaned_ratings['sentiment_score'] = cleaned_ratings.apply(
    lambda row: (get_sentiment_score(row['review/Summary']) +
                 get_sentiment_score(row['review/Text'])) / 2,
    axis=1
)


def categorize_sentiment(score):
    if score >= 0.6:
        return 'excellent'
    elif score >= 0.2:
        return 'good'
    elif score >= -0.2:
        return 'average'
    elif score >= -0.6:
        return 'bad'
    else:
        return 'worse'

cleaned_ratings['sentiment_category'] = cleaned_ratings['sentiment_score'].apply(categorize_sentiment)
cleaned_ratings.head()


# Calculate sentiment category counts
sentiment_counts = cleaned_ratings['sentiment_category'].value_counts()
sentiment_counts

sentiment_category
average      1851532
good          914078
excellent     149626
bad            59815
worse           2142
Name: count, dtype: int64


import matplotlib.pyplot as plt

# Plot the pie chart
plt.figure(figsize=(6, 6))
plt.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', startangle=140, colors=['yellow', 'lightgreen', 'green', 'pink', 'red'])
plt.title('Sentiment Category Distribution')
plt.show()


cleaned_ratings.to_csv("ratings with sentiment analysis.csv")


import numpy as np
import pandas as pd


from textblob import TextBlob

def get_sentiment_score(text):
    # Handle NaN values by converting them to empty strings
    if pd.isna(text):
        text = ""
    analysis = TextBlob(text)
    # The sentiment score ranges from -1 (negative) to 1 (positive)
    return analysis.sentiment.polarity


# Apply sentiment analysis to both 'review/Summary' and 'review/Text'
cleaned_ratings['sentiment_score'] = cleaned_ratings.apply(
    lambda row: (get_sentiment_score(row['review/Summary']) +
                 get_sentiment_score(row['review/Text'])) / 2,
    axis=1
)


def categorize_sentiment(score):
    if score >= 0.6:
        return 'excellent'
    elif score >= 0.2:
        return 'good'
    elif score >= -0.2:
        return 'average'
    elif score >= -0.6:
        return 'bad'
    else:
        return 'worse'

cleaned_ratings['sentiment_category'] = cleaned_ratings['sentiment_score'].apply(categorize_sentiment)
cleaned_ratings.head()


# Calculate sentiment category counts
sentiment_counts = cleaned_ratings['sentiment_category'].value_counts()
sentiment_counts

sentiment_category
average      1851532
good          914078
excellent     149626
bad            59815
worse           2142
Name: count, dtype: int64


import matplotlib.pyplot as plt

# Plot the pie chart
plt.figure(figsize=(6, 6))
plt.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', startangle=140, colors=['yellow', 'lightgreen', 'green', 'pink', 'red'])
plt.title('Sentiment Category Distribution')
plt.show()


cleaned_ratings.to_csv("ratings_with_sentiment_analysis.csv")


cleaned_ratings = pd.read_csv('/kaggle/input/book-ratings-with-sentiment-analysis/ratings with sentiment analysis.csv')
cleaned_ratings.head()


cleaned_ratings.drop('Unnamed: 0', axis = 1, inplace = True)
cleaned_ratings.shape

(2977193, 8)


import pandas as pd
from gensim import corpora
from gensim.models import LdaModel
from nltk.tokenize import word_tokenize
import string

# Tokenize, remove stopwords, and punctuation
def preprocess(text):
    if isinstance(text, list):
        text = ' '.join(text)  # Join list if already tokenized
    if isinstance(text, str):
        tokens = word_tokenize(text.lower())  # Tokenize and lowercase
        tokens = [word for word in tokens if word not in string.punctuation]  # Remove punctuation
        return tokens
    else:
        return []  # Return empty list if input is not string

# Define the size of each chunk (e.g., 50,000 rows per chunk)
chunk_size = 50000

# Split the dataset into chunks
chunks = [cleaned_ratings[i:i + chunk_size] for i in range(0, cleaned_ratings.shape[0], chunk_size)]

# Iterate over each chunk, process it, and store the results
for idx, chunk in enumerate(chunks):
    print(f'Processing chunk {idx + 1} of {len(chunks)}...')

    # Preprocess the text data
    chunk = chunk.copy()  # Make a copy to avoid warnings
    chunk['review/Text'] = chunk['review/Text'].fillna('').apply(preprocess)

    # Filter out rows where 'review/Text' is empty after preprocessing
    chunk = chunk[chunk['review/Text'].apply(len) > 0]

    # Create a dictionary and corpus for the chunk
    dictionary = corpora.Dictionary(chunk['review/Text'])
    dictionary.filter_extremes(no_below=5, no_above=0.5)
    corpus = [dictionary.doc2bow(text) for text in chunk['review/Text']]

    # Ensure the corpus has valid entries
    if len(corpus) > 0:
        # Train the LDA model on the chunk
        lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, random_state=42, passes=10)

        # Get the dominant topic for each review in the chunk
        def get_dominant_topic(lda_model, bow):
            topics = lda_model.get_document_topics(bow)
            return max(topics, key=lambda x: x[1])[0] if topics else None

        chunk['dominant_topic'] = [get_dominant_topic(lda_model, bow) for bow in corpus]

    # Save each chunk's result to a CSV file
    chunk.to_csv(f'chunk_{idx}_results.csv', index=False)

# Combine all chunk results into one DataFrame
combined_df = pd.concat([pd.read_csv(f'chunk_{i}_results.csv') for i in range(len(chunks))], ignore_index=True)

# Save the combined results to a new CSV file
combined_df.to_csv('combined_topic_results.csv', index=False)

# Print a preview of the combined results
print(combined_df[['Title', 'dominant_topic']].head())

Processing chunk 1 of 60...
Processing chunk 2 of 60...
Processing chunk 3 of 60...
Processing chunk 4 of 60...
Processing chunk 5 of 60...
Processing chunk 6 of 60...
Processing chunk 7 of 60...
Processing chunk 8 of 60...
Processing chunk 9 of 60...
Processing chunk 10 of 60...
Processing chunk 11 of 60...
Processing chunk 12 of 60...
Processing chunk 13 of 60...
Processing chunk 14 of 60...
Processing chunk 15 of 60...
Processing chunk 16 of 60...
Processing chunk 17 of 60...
Processing chunk 18 of 60...
Processing chunk 19 of 60...
Processing chunk 20 of 60...
Processing chunk 21 of 60...
Processing chunk 22 of 60...
Processing chunk 23 of 60...
Processing chunk 24 of 60...
Processing chunk 25 of 60...
Processing chunk 26 of 60...
Processing chunk 27 of 60...
Processing chunk 28 of 60...
Processing chunk 29 of 60...
Processing chunk 30 of 60...
Processing chunk 31 of 60...
Processing chunk 32 of 60...
Processing chunk 33 of 60...
Processing chunk 34 of 60...
Processing chunk 35 of 60...
Processing chunk 36 of 60...
Processing chunk 37 of 60...
Processing chunk 38 of 60...
Processing chunk 39 of 60...
Processing chunk 40 of 60...
Processing chunk 41 of 60...
Processing chunk 42 of 60...
Processing chunk 43 of 60...
Processing chunk 44 of 60...
Processing chunk 45 of 60...
Processing chunk 46 of 60...
Processing chunk 47 of 60...
Processing chunk 48 of 60...
Processing chunk 49 of 60...
Processing chunk 50 of 60...
Processing chunk 51 of 60...
Processing chunk 52 of 60...
Processing chunk 53 of 60...
Processing chunk 54 of 60...
Processing chunk 55 of 60...
Processing chunk 56 of 60...
Processing chunk 57 of 60...
Processing chunk 58 of 60...
Processing chunk 59 of 60...
Processing chunk 60 of 60...
                            Title  dominant_topic
0  Its Only Art If Its Well Hung!               2
1        Dr. Seuss: American Icon               1
2        Dr. Seuss: American Icon               4
3        Dr. Seuss: American Icon               4
4        Dr. Seuss: American Icon               1


def get_topic_descriptions(lda_model, num_words=5):
    topic_descriptions = {}
    for topic_id, topic in lda_model.show_topics(formatted=False, num_words=num_words):
        topic_descriptions[topic_id] = " ".join([word for word, prop in topic])
    return topic_descriptions

topic_descriptions = get_topic_descriptions(lda_model)
topic_descriptions

{0: 'jane love like stori one',
 1: 'stori adventur classic one great',
 2: 'novel life stori charact man',
 3: 'war histori american america polit',
 4: 'one use work mani author'}


import pyLDAvis.gensim
import pyLDAvis.gensim_models as gensimvis

# Prepare the visualization data
lda_display = gensimvis.prepare(lda_model, corpus, dictionary, sort_topics=False)
pyLDAvis.display(lda_display)


pyLDAvis.save_html(lda_display, 'lda_visualization.html')


!pip install pyspark

Collecting pyspark
  Downloading pyspark-3.5.2.tar.gz (317.3 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 317.3/317.3 MB 4.8 MB/s eta 0:00:0000:0100:01
  Preparing metadata (setup.py) ... done
Requirement already satisfied: py4j==0.10.9.7 in /opt/conda/lib/python3.10/site-packages (from pyspark) (0.10.9.7)
Building wheels for collected packages: pyspark
  Building wheel for pyspark (setup.py) ... done
  Created wheel for pyspark: filename=pyspark-3.5.2-py2.py3-none-any.whl size=317812364 sha256=9fb1c60dd4758f6717eff88dc29809954bdaf5292959ca220a4fbccd4f4ee1ce
  Stored in directory: /root/.cache/pip/wheels/34/34/bd/03944534c44b677cd5859f248090daa9fb27b3c8f8e5f49574
Successfully built pyspark
Installing collected packages: pyspark
Successfully installed pyspark-3.5.2


from pyspark.sql import SparkSession
from pyspark.ml.feature import NGram
from pyspark.sql.functions import col, udf
from pyspark.sql.types import ArrayType, StringType, IntegerType
from pyspark.rdd import RDD

# Initialize SparkSession
spark = SparkSession.builder.appName("NGramAnalysis").getOrCreate()

# Load CSV into DataFrame
df_reviews_with_sentiment = spark.read.csv("/kaggle/input/reviews-with-sentiment-for-trigram",
                                          header=True, inferSchema=True)
df_reviews_with_sentiment.show(5)

+--------------------+-----+------------+-------------------------+----------------------+---------------+------------------------+
|               Title|Price|review/score|Lemmatized review/summary|Lemmatized review/text|Sentiment_Score|Sentiment_Category_VADER|
+--------------------+-----+------------+-------------------------+----------------------+---------------+------------------------+
|Its Only Art If I...| NULL|         4.0|     nice collection j...|  julie strain fans...|         0.9481|                  Better|
|Dr. Seuss: Americ...| NULL|         5.0|             really enjoy|  do not care much ...|          0.984|                  Better|
|Dr. Seuss: Americ...| NULL|         5.0|     essential every p...|  people become boo...|         0.9911|                  Better|
|Dr. Seuss: Americ...| NULL|         4.0|     phlip nel give si...|  theodore seuss ge...|         0.9678|                  Better|
|Dr. Seuss: Americ...| NULL|         4.0|     good academic ove...|  philip nel dr seu...|         0.9697|                  Better|
+--------------------+-----+------------+-------------------------+----------------------+---------------+------------------------+
only showing top 5 rows


from pyspark.sql.functions import udf
from pyspark.sql.types import ArrayType, StringType
import re
from nltk.corpus import stopwords

# Load stopwords
stop_words = set(stopwords.words('english'))

# Define preprocessing function for LDA
def preprocess_for_lda(text):
    # Handle non-string entries by converting to empty string if necessary
    if not isinstance(text, str):
        text = ''

    # Tokenize and remove stopwords
    tokens = text.lower().split()
    tokens = [word for word in tokens if word not in stop_words]

    # Optionally, clean tokens further (remove punctuation, etc.)
    tokens = [re.sub(r'\W+', '', token) for token in tokens]  # Remove special characters
    return tokens

# Register the function as a UDF
preprocess_udf = udf(preprocess_for_lda, ArrayType(StringType()))

# Apply preprocessing using the UDF
df_reviews_with_sentiment = df_reviews_with_sentiment.withColumn('tokens', preprocess_udf(df_reviews_with_sentiment['Lemmatized review/summary']))

# Show the result
df_reviews_with_sentiment.select('Lemmatized review/summary', 'tokens').show(5, truncate=False)

+--------------------------------------------+----------------------------------------------------+
|Lemmatized review/summary                   |tokens                                              |
+--------------------------------------------+----------------------------------------------------+
|nice collection julie strain image          |[nice, collection, julie, strain, image]            |
|really enjoy                                |[really, enjoy]                                     |
|essential every personal public library     |[essential, every, personal, public, library]       |
|phlip nel give silly seuss serious treatment|[phlip, nel, give, silly, seuss, serious, treatment]|
|good academic overview                      |[good, academic, overview]                          |
+--------------------------------------------+----------------------------------------------------+
only showing top 5 rows


df_reviews_with_sentiment.show(5)

+--------------------+-----+------------+-------------------------+----------------------+---------------+------------------------+--------------------+
|               Title|Price|review/score|Lemmatized review/summary|Lemmatized review/text|Sentiment_Score|Sentiment_Category_VADER|              tokens|
+--------------------+-----+------------+-------------------------+----------------------+---------------+------------------------+--------------------+
|Its Only Art If I...| NULL|         4.0|     nice collection j...|  julie strain fans...|         0.9481|                  Better|[nice, collection...|
|Dr. Seuss: Americ...| NULL|         5.0|             really enjoy|  do not care much ...|          0.984|                  Better|     [really, enjoy]|
|Dr. Seuss: Americ...| NULL|         5.0|     essential every p...|  people become boo...|         0.9911|                  Better|[essential, every...|
|Dr. Seuss: Americ...| NULL|         4.0|     phlip nel give si...|  theodore seuss ge...|         0.9678|                  Better|[phlip, nel, give...|
|Dr. Seuss: Americ...| NULL|         4.0|     good academic ove...|  philip nel dr seu...|         0.9697|                  Better|[good, academic, ...|
+--------------------+-----+------------+-------------------------+----------------------+---------------+------------------------+--------------------+
only showing top 5 rows


# Generate trigrams
ngram_transformer_trigram = NGram(n=3, inputCol="tokens", outputCol="trigrams")
df_trigrams = ngram_transformer_trigram.transform(df_reviews_with_sentiment)


def extract_ngrams(row, col_name):
    ngrams = row[col_name]
    if ngrams:
        return [(ngram, 1) for ngram in ngrams]
    else:
        return []

# Convert to RDD and extract trigrams from the DataFrame (replace `df_trigrams` with the actual DataFrame name you have)
trigrams_rdd = df_trigrams.rdd.flatMap(lambda row: extract_ngrams(row, "trigrams"))

# Count the frequency of trigrams
trigram_freq = trigrams_rdd.reduceByKey(lambda a, b: a + b)

# Use `takeOrdered` to get the top 50 most frequent trigrams
top_trigrams = trigram_freq.takeOrdered(50, key=lambda x: -x[1])

# Print the top 50 most common Trigrams
print("Top 50 Trigrams:", top_trigrams)

[Stage 20:================================================>       (12 + 2) / 14]

Top 50 Trigrams: [('good book ever', 14576), ('book ever read', 13307), ('one good book', 6910), ('book ever write', 4028), ('good book read', 3265), ('bad book ever', 2038), ('one favorite book', 1735), ('great book great', 1717), ('great book ever', 1697), ('must read book', 1409), ('book change life', 1359), ('favorite book time', 1280), ('novel ever write', 1244), ('one great book', 1201), ('great american novel', 1114), ('another great book', 1065), ('free sf reader', 1052), ('brave new world', 1031), ('book must read', 1028), ('well worth read', 1003), ('must read anyone', 916), ('great book read', 895), ('good novel ever', 864), ('really good book', 849), ('love love love', 820), ('could put book', 800), ('picture dorian gray', 779), ('well write book', 743), ('pretty good book', 727), ('read long time', 713), ('favorite book ever', 713), ('book read year', 705), ('novel ever read', 685), ('lion witch wardrobe', 638), ('one time favorite', 633), ('one alltime favorite', 632), ('must read every', 609), ('waste time money', 594), ('alice adventure wonderland', 586), ('tour de force', 575), ('stand test time', 568), ('good book good', 560), ('tree grow brooklyn', 544), ('count monte cristo', 539), ('funniest book ever', 530), ('real page turner', 527), ('book worth read', 527), ('good book world', 524), ('world war ii', 521), ('one good novel', 517)]


import matplotlib.pyplot as plt
import squarify

# Example list of top 50 N-grams with their frequencies
top_trigrams = [('good book ever', 14576), ('book ever read', 13307), ('one good book', 6910), ('book ever write', 4028),
                ('good book read', 3265), ('bad book ever', 2038), ('one favorite book', 1735), ('great book great', 1717),
                ('great book ever', 1697), ('must read book', 1409), ('book change life', 1359), ('favorite book time', 1280),
                ('novel ever write', 1244), ('one great book', 1201), ('great american novel', 1114), ('another great book', 1065),
                ('free sf reader', 1052), ('brave new world', 1031), ('book must read', 1028), ('well worth read', 1003),
                ('must read anyone', 916), ('great book read', 895), ('good novel ever', 864), ('really good book', 849),
                ('love love love', 820), ('could put book', 800), ('picture dorian gray', 779), ('well write book', 743),
                ('pretty good book', 727), ('read long time', 713), ('favorite book ever', 713), ('book read year', 705),
                ('novel ever read', 685), ('lion witch wardrobe', 638), ('one time favorite', 633), ('one alltime favorite', 632),
                ('must read every', 609), ('waste time money', 594), ('alice adventure wonderland', 586), ('tour de force', 575),
                ('stand test time', 568), ('good book good', 560), ('tree grow brooklyn', 544), ('count monte cristo', 539),
                ('funniest book ever', 530), ('real page turner', 527), ('book worth read', 527), ('good book world', 524),
                ('world war ii', 521), ('one good novel', 517)]

# Extract N-grams and their frequencies
ngrams_texts = [ngram for ngram, count in top_trigrams]
frequencies = [count for ngram, count in top_trigrams]

# Combine the N-gram text with its frequency for labeling
labels = [f'{ngram}\n{count}' for ngram, count in top_trigrams]

# Create a treemap
plt.figure(figsize=(30, 15))
squarify.plot(sizes=frequencies, label=labels, alpha=0.8, color=plt.cm.Paired.colors)

# Customize the plot
plt.title('Top 50 N-grams Treemap in Book Reviews')
plt.axis('off')  # Turn off the axes
plt.show()


import pandas as pd
import numpy as np


df_reviews_with_sentiment = pd.read_csv("/kaggle/input/merged-data/merged data (1).csv")
df_reviews_with_sentiment.head()


df_reviews_with_sentiment.shape

(40634, 8)


df_reviews_with_sentiment.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 40634 entries, 0 to 40633
Data columns (total 8 columns):
 #   Column         Non-Null Count  Dtype  
---  ------         --------------  -----  
 0   Unnamed: 0     40634 non-null  int64  
 1   Title          40583 non-null  object 
 2   authors        40634 non-null  object 
 3   publisher      40634 non-null  object 
 4   publishedYear  40634 non-null  int64  
 5   categories     40634 non-null  object 
 6   description    40617 non-null  object 
 7   hybrid_score   40634 non-null  float64
dtypes: float64(1), int64(2), object(5)
memory usage: 2.5+ MB


df_reviews_with_sentiment.isnull().sum()

Unnamed: 0        0
Title            51
authors           0
publisher         0
publishedYear     0
categories        0
description      17
hybrid_score      0
dtype: int64


# Remove rows where the null value count is insignificant
df_reviews_with_sentiment.dropna(subset=['Title', 'description'], inplace = True)

missing_values = df_reviews_with_sentiment.isnull().sum()
print("Missing values:\n", missing_values)

Missing values:
 Unnamed: 0       0
Title            0
authors          0
publisher        0
publishedYear    0
categories       0
description      0
hybrid_score     0
dtype: int64


df_reviews_with_sentiment.shape

(40566, 8)


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
from scipy.sparse import vstack


# Define chunk size
chunk_size = 1000  

# Initialize variables to store results
cosine_sim_list = []

# Convert lemmatized reviews to TF-IDF features
vectorizer = TfidfVectorizer(stop_words='english')

# Handle missing values in 'Lemmatized review/text' (replace NaN with empty strings)
df_reviews_with_sentiment['description'] = df_reviews_with_sentiment['description'].fillna('')

# Adjust the length to exclude the last chunk if it's smaller than chunk_size
adjusted_len = len(df_reviews_with_sentiment) // chunk_size * chunk_size

# Fit the vectorizer on the entire dataset to ensure consistent vocabulary
print("Fitting the TF-IDF vectorizer on the entire dataset...")
vectorizer.fit(df_reviews_with_sentiment['description'])

# Process the dataset in chunks
num_chunks = adjusted_len // chunk_size
print(f"Processing dataset in {num_chunks} chunks of size {chunk_size}, ignoring the last {len(df_reviews_with_sentiment) - adjusted_len} rows...")

# Process data in full-size chunks
for i in range(num_chunks):
    # Define the range for this chunk
    start_idx = i * chunk_size
    end_idx = (i + 1) * chunk_size
    
    # Extract the chunk of text data
    chunk = df_reviews_with_sentiment['description'].iloc[start_idx:end_idx]
    
    # Convert to TF-IDF features for this chunk using the pre-fitted vectorizer
    print(f"Chunk {i+1}/{num_chunks}: Converting text to TF-IDF...")
    tfidf_chunk = vectorizer.transform(chunk)
    
    # Store the TF-IDF matrix of this chunk
    cosine_sim_list.append(tfidf_chunk)
    
    # Provide feedback after processing each chunk
    print(f"Chunk {i+1}/{num_chunks} processed and stored.\n")

# Combine all TF-IDF matrices into one large matrix
print("Combining all chunks into the final TF-IDF matrix...")
tfidf_matrix = vstack(cosine_sim_list)

print("TF-IDF matrix computed successfully.\n")

# Compute the cosine similarity matrix
print("Computing cosine similarity matrix...")
cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)

print("Cosine similarity matrix computed successfully.\n")

# Create the recommendation function
def recommend_books_with_sentiment(title, cosine_sim=cosine_sim, df=df_reviews_with_sentiment):
    idx = df.index[df['Title'] == title].tolist()[0]
    
    # Get the pairwise similarity scores of all books with that book
    sim_scores = list(enumerate(cosine_sim[idx]))
    
    # Multiply similarity scores by the sentiment score of each book to weight the recommendations
    sim_scores = [(i, score * df['hybrid_score'].iloc[i]) for i, score in sim_scores]
    
    # Sort the books based on similarity and sentiment scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    
    # Get the indices of the most similar books (excluding the queried book itself)
    sim_scores = sim_scores[1:11]
    book_indices = [i[0] for i in sim_scores]
    
    # Return the titles of the top 10 most similar books
    return df['Title'].iloc[book_indices]

# Example usage 1
print("Recommending books...")
recommended_books = recommend_books_with_sentiment('battleship bismarck')
print(recommended_books)

Fitting the TF-IDF vectorizer on the entire dataset...
Processing dataset in 40 chunks of size 1000, ignoring the last 566 rows...
Chunk 1/40: Converting text to TF-IDF...
Chunk 1/40 processed and stored.

Chunk 2/40: Converting text to TF-IDF...
Chunk 2/40 processed and stored.

Chunk 3/40: Converting text to TF-IDF...
Chunk 3/40 processed and stored.

Chunk 4/40: Converting text to TF-IDF...
Chunk 4/40 processed and stored.

Chunk 5/40: Converting text to TF-IDF...
Chunk 5/40 processed and stored.

Chunk 6/40: Converting text to TF-IDF...
Chunk 6/40 processed and stored.

Chunk 7/40: Converting text to TF-IDF...
Chunk 7/40 processed and stored.

Chunk 8/40: Converting text to TF-IDF...
Chunk 8/40 processed and stored.

Chunk 9/40: Converting text to TF-IDF...
Chunk 9/40 processed and stored.

Chunk 10/40: Converting text to TF-IDF...
Chunk 10/40 processed and stored.

Chunk 11/40: Converting text to TF-IDF...
Chunk 11/40 processed and stored.

Chunk 12/40: Converting text to TF-IDF...
Chunk 12/40 processed and stored.

Chunk 13/40: Converting text to TF-IDF...
Chunk 13/40 processed and stored.

Chunk 14/40: Converting text to TF-IDF...
Chunk 14/40 processed and stored.

Chunk 15/40: Converting text to TF-IDF...
Chunk 15/40 processed and stored.

Chunk 16/40: Converting text to TF-IDF...
Chunk 16/40 processed and stored.

Chunk 17/40: Converting text to TF-IDF...
Chunk 17/40 processed and stored.

Chunk 18/40: Converting text to TF-IDF...
Chunk 18/40 processed and stored.

Chunk 19/40: Converting text to TF-IDF...
Chunk 19/40 processed and stored.

Chunk 20/40: Converting text to TF-IDF...
Chunk 20/40 processed and stored.

Chunk 21/40: Converting text to TF-IDF...
Chunk 21/40 processed and stored.

Chunk 22/40: Converting text to TF-IDF...
Chunk 22/40 processed and stored.

Chunk 23/40: Converting text to TF-IDF...
Chunk 23/40 processed and stored.

Chunk 24/40: Converting text to TF-IDF...
Chunk 24/40 processed and stored.

Chunk 25/40: Converting text to TF-IDF...
Chunk 25/40 processed and stored.

Chunk 26/40: Converting text to TF-IDF...
Chunk 26/40 processed and stored.

Chunk 27/40: Converting text to TF-IDF...
Chunk 27/40 processed and stored.

Chunk 28/40: Converting text to TF-IDF...
Chunk 28/40 processed and stored.

Chunk 29/40: Converting text to TF-IDF...
Chunk 29/40 processed and stored.

Chunk 30/40: Converting text to TF-IDF...
Chunk 30/40 processed and stored.

Chunk 31/40: Converting text to TF-IDF...
Chunk 31/40 processed and stored.

Chunk 32/40: Converting text to TF-IDF...
Chunk 32/40 processed and stored.

Chunk 33/40: Converting text to TF-IDF...
Chunk 33/40 processed and stored.

Chunk 34/40: Converting text to TF-IDF...
Chunk 34/40 processed and stored.

Chunk 35/40: Converting text to TF-IDF...
Chunk 35/40 processed and stored.

Chunk 36/40: Converting text to TF-IDF...
Chunk 36/40 processed and stored.

Chunk 37/40: Converting text to TF-IDF...
Chunk 37/40 processed and stored.

Chunk 38/40: Converting text to TF-IDF...
Chunk 38/40 processed and stored.

Chunk 39/40: Converting text to TF-IDF...
Chunk 39/40 processed and stored.

Chunk 40/40: Converting text to TF-IDF...
Chunk 40/40 processed and stored.

Combining all chunks into the final TF-IDF matrix...
TF-IDF matrix computed successfully.

Computing cosine similarity matrix...
Cosine similarity matrix computed successfully.

Recommending books...
2                            battleship bismarck
36106                               sink bismark
38599                            alarm starboard
37358                              sink bismarch
36864                                    titanic
30218                          titanic end dream
19641                              ship must die
31285                       destruction bismarck
29965    wreck sink titanic ocean great disaster
13648                                 battleship
Name: Title, dtype: object


print("Recommending books...")
recommended_books = recommend_books_with_sentiment('beginner yoruba hippocrene beginners series')
print(recommended_books)

Recommending books...
26488    good bad beautiful discourse value yoruba culture
25678    danish compact learn speak understand danish p...
39388    sixteen cowrie yoruba divination africa new world
667      japanese everyone functional approach daily co...
30198    history yorubas early time begin british prote...
36545    history yorubas early time begin british prote...
32908    integrate korean advanced intermediate klear t...
28714                            beginner chinese audio cd
11574                           german child passport book
36952                                          learn speak
Name: Title, dtype: object


print("Recommending books...")
recommended_books = recommend_books_with_sentiment('tess highlander')
print(recommended_books)

Recommending books...
11731                           dance away big book series
36412                                      perfect husband
8515     perfect husband fairy tale demystify marriage ...
36999                                     last place novel
16384                 sweet revenge harlequin love laugher
1661                                     bad boy black tie
8528                                              dear zoe
33942                                             tidepool
38017    flax super food delicious recipe use flax oil ...
7864       bard odyssey irish celtic world morgan llywelyn
Name: Title, dtype: object


pip install ipywidgets

Note: you may need to restart the kernel to use updated packages.


# Network Graph for recommendations with sentiment incorporated
import networkx as nx
import matplotlib.pyplot as plt

def plot_recommendations_with_sentiment(title, cosine_sim=cosine_sim, df=df_reviews_with_sentiment):
    idx = df.index[df['Title'] == title].tolist()[0]
    sim_scores = list(enumerate(cosine_sim[idx]))
    
    # Adjust similarity by sentiment score
    sim_scores = [(i, score * df['hybrid_score'].iloc[i]) for i, score in sim_scores]
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:11]
    book_indices = [i[0] for i in sim_scores]
    similar_books = df.iloc[book_indices]

    # Build the network graph
    G = nx.Graph()
    for i, row in similar_books.iterrows():
        G.add_node(row['Title'])
    
    for i, (j, score) in enumerate(sim_scores):
        if i > 0:
            G.add_edge(df.iloc[idx]['Title'], df.iloc[j]['Title'], weight=score)
    
    # Plot the graph
    pos = nx.spring_layout(G, seed=42)
    plt.figure(figsize=(8, 6))
    nx.draw(G, pos, with_labels=True, node_color='lightblue', node_size=3000, edge_color='gray', font_size=12, font_weight='bold')
    plt.title('Book Recommendations Network with Sentiment')
    plt.show()

plot_recommendations_with_sentiment('battleship bismarck')


plot_recommendations_with_sentiment('rabbis cat')


plot_recommendations_with_sentiment('zombie lover xanth')


plot_recommendations_with_sentiment('burn')


plot_recommendations_with_sentiment('titanic')


import networkx as nx
import matplotlib.pyplot as plt
import ipywidgets as widgets
from IPython.display import display

def plot_recommendations_with_sentiment(title, cosine_sim=cosine_sim, df=df_reviews_with_sentiment):
    idx = df.index[df['Title'] == title].tolist()[0]
    sim_scores = list(enumerate(cosine_sim[idx]))
    
    # Adjust similarity by sentiment score
    sim_scores = [(i, score * df['hybrid_score'].iloc[i]) for i, score in sim_scores]
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:11]
    book_indices = [i[0] for i in sim_scores]
    similar_books = df.iloc[book_indices]

    # Build the network graph
    G = nx.Graph()
    for i, row in similar_books.iterrows():
        G.add_node(row['Title'])
    
    for i, (j, score) in enumerate(sim_scores):
        if i > 0:
            G.add_edge(df.iloc[idx]['Title'], df.iloc[j]['Title'], weight=score)
    
    # Plot the graph
    pos = nx.spring_layout(G, seed=42)
    plt.figure(figsize=(8, 6))
    nx.draw(G, pos, with_labels=True, node_color='lightblue', node_size=3000, edge_color='gray', font_size=12, font_weight='bold')
    plt.title('Book Recommendations Network with Sentiment')
    plt.show()

# Create a text input widget for book title
title_input = widgets.Text(
    description='Book Title:',
    value='battleship bismarck',  # Default value
    style={'description_width': 'initial'}
)

# Define a function to update the plot based on the input title
def update_graph(title):
    plot_recommendations_with_sentiment(title)

# Link the text input widget to the update function
widgets.interactive(update_graph, title=title_input)

interactive(children=(Text(value='battleship bismarck', description='Book Title:', style=DescriptionStyle(desc…

	Title	description	authors	image	previewLink	publisher	publishedDate	infoLink	categories	ratingsCount
0	Its Only Art If Its Well Hung!	NaN	['Julie Strain']	http://books.google.com/books/content?id=DykPA...	http://books.google.nl/books?id=DykPAAAACAAJ&d...	NaN	1996	http://books.google.nl/books?id=DykPAAAACAAJ&d...	['Comics & Graphic Novels']	NaN
1	Dr. Seuss: American Icon	Philip Nel takes a fascinating look into the k...	['Philip Nel']	http://books.google.com/books/content?id=IjvHQ...	http://books.google.nl/books?id=IjvHQsCn_pgC&p...	A&C Black	2005-01-01	http://books.google.nl/books?id=IjvHQsCn_pgC&d...	['Biography & Autobiography']	NaN
2	Wonderful Worship in Smaller Churches	This resource includes twelve principles in un...	['David R. Ray']	http://books.google.com/books/content?id=2tsDA...	http://books.google.nl/books?id=2tsDAAAACAAJ&d...	NaN	2000	http://books.google.nl/books?id=2tsDAAAACAAJ&d...	['Religion']	NaN
3	Whispers of the Wicked Saints	Julia Thomas finds her life spinning out of co...	['Veronica Haddon']	http://books.google.com/books/content?id=aRSIg...	http://books.google.nl/books?id=aRSIgJlq6JwC&d...	iUniverse	2005-02	http://books.google.nl/books?id=aRSIgJlq6JwC&d...	['Fiction']	NaN
4	Nation Dance: Religion, Identity and Cultural ...	NaN	['Edward Long']	NaN	http://books.google.nl/books?id=399SPgAACAAJ&d...	NaN	2003-03-01	http://books.google.nl/books?id=399SPgAACAAJ&d...	NaN	NaN

	Title	description	authors	publisher	publishedDate	categories	ratingsCount
0	Its Only Art If Its Well Hung!	NaN	['Julie Strain']	NaN	1996	['Comics & Graphic Novels']	NaN
1	Dr. Seuss: American Icon	Philip Nel takes a fascinating look into the k...	['Philip Nel']	A&C Black	2005-01-01	['Biography & Autobiography']	NaN
2	Wonderful Worship in Smaller Churches	This resource includes twelve principles in un...	['David R. Ray']	NaN	2000	['Religion']	NaN
3	Whispers of the Wicked Saints	Julia Thomas finds her life spinning out of co...	['Veronica Haddon']	iUniverse	2005-02	['Fiction']	NaN
4	Nation Dance: Religion, Identity and Cultural ...	NaN	['Edward Long']	NaN	2003-03-01	NaN	NaN

	Title	description	authors	publisher	publishedDate	categories	ratingsCount	publishedYear
5	The Church of Christ: A Biblical Ecclesiology ...	In The Church of Christ: A Biblical Ecclesiolo...	['Everett Ferguson']	Wm. B. Eerdmans Publishing	1996	['Religion']	5.0	1996
31	Voices from the Farm: Adventures in Community ...	Twenty-five years ago, at the height of the co...	['Rupert Fike']	Book Publishing Company	2012-08-21	['Biography & Autobiography']	1.0	2012
33	The Battleship Bismarck	The Bismarck is perhaps the most famous – and ...	['Stefan Draminski']	Bloomsbury Publishing	2018-09-20	['History']	1.0	2018
42	Tess and the Highlander	In 1543, on a windswept isle off of Scotland, ...	['May Mcgoldrick']	Harper Collins	2002-11	['Juvenile Fiction']	2.0	2002
43	Beginner's Yoruba (Hippocrene Beginner's Series)	"Beginner's Yoruba" is now available with two ...	['Kayode J. Fakinlede']	Hippocrene Books	2005	['Foreign Language Study']	1.0	2005

	Title	description	authors	publisher	categories	ratingsCount	publishedYear
5	The Church of Christ: A Biblical Ecclesiology ...	In The Church of Christ: A Biblical Ecclesiolo...	['Everett Ferguson']	Wm. B. Eerdmans Publishing	['Religion']	5.0	1996
31	Voices from the Farm: Adventures in Community ...	Twenty-five years ago, at the height of the co...	['Rupert Fike']	Book Publishing Company	['Biography & Autobiography']	1.0	2012
33	The Battleship Bismarck	The Bismarck is perhaps the most famous – and ...	['Stefan Draminski']	Bloomsbury Publishing	['History']	1.0	2018
42	Tess and the Highlander	In 1543, on a windswept isle off of Scotland, ...	['May Mcgoldrick']	Harper Collins	['Juvenile Fiction']	2.0	2002
43	Beginner's Yoruba (Hippocrene Beginner's Series)	"Beginner's Yoruba" is now available with two ...	['Kayode J. Fakinlede']	Hippocrene Books	['Foreign Language Study']	1.0	2005

	count
publishedYear
1776	3
1823	1
1852	1
1861	1
1869	1
...	...
2019	604
2020	648
2021	618
2022	241
2023	3

Preprocessing¶

Sentiment Analysis¶

Sentiment Analysis¶

Latent Dirichlet Allocation (LDA)¶

3. N-gram Analysis¶

4. Product Recommendation System¶

Content-Based Filtering with Sentiment Labels¶

	Title	description
5	The Church of Christ: A Biblical Ecclesiology ...	in the church of christ a biblical ecclesiolog...
31	Voices from the Farm: Adventures in Community ...	twentyfive year ago at the height of the count...
33	The Battleship Bismarck	the bismarck is perhaps the most famous – and ...
42	Tess and the Highlander	in 1543 on a windswept isle off of scotland se...
43	Beginner's Yoruba (Hippocrene Beginner's Series)	beginner yoruba is now available with two acco...

	Category	Count
14	Fiction	11149
4	Juvenile Fiction	3346
0	Religion	2902
3	History	2465
1	Biography	2312
2	Autobiography	2309
21	Business	1064
22	Economics	1061
23	Juvenile Nonfiction	1019
39	Computers	881

	Author	Count
0	"Louis LAmour"	98
1	Agatha Christie	56
2	Nora Roberts	48
3	Georgette Heyer	43
4	William Shakespeare	40
...	...	...
32593	Iris Rainer Dart	1
32594	Richard Hunt	1
32595	Henry Justice Ford	1
32596	Dave Grohl	1
32597	Elvira Woodruff	1

	Id	Title	Price	User_id	profileName	review/helpfulness	review/score	review/time	review/summary	review/text
0	1882931173	Its Only Art If Its Well Hung!	NaN	AVCGYZL8FQQTD	Jim of Oz "jim-of-oz"	7/7	4.0	940636800	Nice collection of Julie Strain images	This is only for Julie Strain fans. It's a col...
1	0826414346	Dr. Seuss: American Icon	NaN	A30TK6U7DNS82R	Kevin Killian	10/10	5.0	1095724800	Really Enjoyed It	I don't care much for Dr. Seuss but after read...
2	0826414346	Dr. Seuss: American Icon	NaN	A3UH4UZ4RSVO82	John Granger	10/11	5.0	1078790400	Essential for every personal and Public Library	If people become the books they read and if "t...
3	0826414346	Dr. Seuss: American Icon	NaN	A2MVUWT453QH61	Roy E. Perry "amateur philosopher"	7/7	4.0	1090713600	Phlip Nel gives silly Seuss a serious treatment	Theodore Seuss Geisel (1904-1991), aka "D...
4	0826414346	Dr. Seuss: American Icon	NaN	A22X4XUPKF66MR	D. H. Richards "ninthwavestore"	3/3	4.0	1107993600	Good academic overview	Philip Nel - Dr. Seuss: American IconThis is b...

	Title	Price	review/score	review/Summary	review/Text
0	Its Only Art If Its Well Hung!	NaN	4.0	nice collect juli strain imag	juli strain fan collect photo page worth nice ...
1	Dr. Seuss: American Icon	NaN	5.0	realli enjoy	dont care much dr seuss read philip nel book c...
2	Dr. Seuss: American Icon	NaN	5.0	essenti everi person public librari	peopl becom book read child father man dr seus...
3	Dr. Seuss: American Icon	NaN	4.0	phlip nel give silli seuss seriou treatment	theodor seuss geisel aka quotdr seussquot one ...
4	Dr. Seuss: American Icon	NaN	4.0	good academ overview	philip nel dr seuss american iconthi basic aca...

	Unnamed: 0	Title	authors	publisher	publishedYear	categories	description	hybrid_score
0	0	church christ biblical ecclesiology today	['Everett Ferguson']	Wm. B. Eerdmans Publishing	1996	['Religion']	church christ biblical ecclesiology today resp...	3.993130
1	1	voice farm adventure community live	['Rupert Fike']	Book Publishing Company	2012	['Biography & Autobiography']	twentyfive year ago height counterculture move...	4.203766
2	2	battleship bismarck	['Stefan Draminski']	Bloomsbury Publishing	2018	['History']	bismarck perhaps famous notorious warship ever...	3.413042
3	3	tess highlander	['May Mcgoldrick']	Harper Collins	2002	['Juvenile Fiction']	windswept isle scotland seventeenyearold tess ...	3.942543
4	4	beginner yoruba hippocrene beginners series	['Kayode J. Fakinlede']	Hippocrene Books	2005	['Foreign Language Study']	beginner yoruba available two accompany audio ...	3.838243